{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 导入必要的工具包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import roc_auc_score\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 读取及观察数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_train = pd.read_csv('./桌面/data/RentListingInquries_FE_train.csv')\n",
    "data_test = pd.read_csv('./桌面/data/RentListingInquries_FE_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 49352 entries, 0 to 49351\n",
      "Columns: 228 entries, bathrooms to interest_level\n",
      "dtypes: float64(9), int64(219)\n",
      "memory usage: 85.8 MB\n"
     ]
    }
   ],
   "source": [
    "data_train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "bathrooms                 0\n",
       "bedrooms                  0\n",
       "price                     0\n",
       "price_bathrooms           0\n",
       "price_bedrooms            0\n",
       "room_diff                 0\n",
       "room_num                  0\n",
       "Year                      0\n",
       "Month                     0\n",
       "Day                       0\n",
       "Wday                      0\n",
       "Yday                      0\n",
       "hour                      0\n",
       "top_10_manager            0\n",
       "top_25_manager            0\n",
       "top_5_manager             0\n",
       "top_50_manager            0\n",
       "top_1_manager             0\n",
       "top_2_manager             0\n",
       "top_15_manager            0\n",
       "top_20_manager            0\n",
       "top_30_manager            0\n",
       "cenroid                   0\n",
       "distance                  0\n",
       "display_address_pred_0    0\n",
       "display_address_pred_1    0\n",
       "display_address_pred_2    0\n",
       "1br                       0\n",
       "24                        0\n",
       "2br                       0\n",
       "                         ..\n",
       "sublet                    0\n",
       "subway                    0\n",
       "super                     0\n",
       "superintendent            0\n",
       "swimming                  0\n",
       "tenant                    0\n",
       "term                      0\n",
       "terrace                   0\n",
       "time                      0\n",
       "tons                      0\n",
       "train                     0\n",
       "true                      0\n",
       "tv                        0\n",
       "unit                      0\n",
       "utilities                 0\n",
       "valet                     0\n",
       "video                     0\n",
       "view                      0\n",
       "views                     0\n",
       "virtual                   0\n",
       "walk                      0\n",
       "walls                     0\n",
       "war                       0\n",
       "washer                    0\n",
       "water                     0\n",
       "wheelchair                0\n",
       "wifi                      0\n",
       "windows                   0\n",
       "work                      0\n",
       "interest_level            0\n",
       "Length: 228, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_train.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(49352, 228)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 2, 0])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_train['interest_level'].unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 分离目标值和特征值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = data_train.drop(columns=['interest_level'])\n",
    "y_train = data_train['interest_level']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 初步确定弱学习数目"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "from xgboost.sklearn import XGBClassifier\n",
    "xgb1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=1000,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.8,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_param = xgb1.get_xgb_params()\n",
    "#三中分类的问题，把num_class 设置为3\n",
    "xgb_param['num_class'] = 3\n",
    "#通过Dmatrix创建xgboost 需要的数据格式\n",
    "xgtrain = xgb.DMatrix(x_train, label=y_train) \n",
    "#通过xgboost 中内置的交叉验证选择初始的若学习期数目\n",
    "cvresult_round1 = xgb.cv(xgb_param, xgtrain, num_boost_round=xgb1.get_params()['n_estimators'], folds =kfold,metrics='mlogloss', early_stopping_rounds=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "cvresult_round1.to_csv('xgBoostRentalCV_Result.csv',index_label = 'n_estimators')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmcXFWZ//HPU9VbesvWDWRfIAQCYsSwqggiY0AEBURQHMGF0ZFBnXEcGP0xiIOCMy6IjAoMMC7AgIoiIIrsOggECEsCWQhZOgnZO51OJ53uruf3xznVXen0Ulmqq7rr+3697qvr3nvuvU/f6q6nzjn3nmvujoiICEAi3wGIiEjhUFIQEZFOSgoiItJJSUFERDopKYiISCclBRER6aSkIJLBzP7VzG7Odxwi+aKkMMiYWbWZLTWzj2YsqzGz5WZ2TsayWWZ2n5ltMrNGM5tvZleb2ci4/kIz6zCz5jgtMbPP5Tj2E82sIZfH2B09xePu33T3T+foeEvN7L252HcuDNT7NdjOy1CnpDDIuHszcDFwnZnVx8XfBua4+y8BzOx44DHgL8Ah7j4CmA20A2/N2N1T7l7t7tXAOcC3zextA/ObyO4ws5J8xyBFwt01DcIJuA24AzgR2ACMyVj3Z+D6fra/EPhzt2XPAB/NmD8DmAc0EpLMoRnrDo3LGmOZMzLWnQbMB7YAK4EvA1XANiAFNMdpbC+/1w3A/XH7p4EDszgfhwAPARuBBcC5exIPcCXw87jdZMCBi4AVwCbgs8BRwEvxd/9hxnEOBB6J78d64BfAiLjuZ/FY2+KxvpLFOV4K/Es8VitQEudXxt9lAXByD+fiWOBNIJmx7EPAS/H10cAcoAlYA3y3l3N6ItDQy7rhwE+BdcAy4GtAIq5LAt+J5+AN4JJ4Hkt62ddS4L29rPsMsDi+r/em/2YAA74HrAU2x3N0eG/vd77/XwfTlPcANO3hGwcjgdXxH++ijOVVQAdwYj/bX0hGUogfdI3AwXH+YGArcApQCnwl/nOWxfnFwL/G+ffEf8DpcdvVwLsy4jwyvu71QyYjjtviB8DR8UPwF8Cd/WxTRfjQvihuc2Q8L4ftbjz0nBR+DFQAfwNsB34D7AeMix9K747lD4rnqxyoB54Avp+x750+/Po6xxnl5wITgGHA9Ph7js2Ir8eECbwOnJIxfzdwWXz9FPDx+LoaOLaXffT6fhESwm+BmhjHQuBTcd1nCR/K4+P5/hN7kBTi39X6+H6WA9cDT8R17wOeA0YQEsShxC9Gvb3fmrKb1Hw0SLn7JsI3zErg1xmrRhKaBd9MLzCzb8d+ha1m9rWMssfG5c2EWsLPgEVx3UeA+939IXdvA/6T8MF0POGbaDVwjbvvcPdHgPuA8+O2bcAMM6t1903u/vxu/nq/dvdn3L2dkBRm9lP+dGCpu9/q7u3xeL8iNInti3i+4e7b3f2PhA/xO9x9rbuvBJ4E3gbg7ovj+Wp193XAd4F397Hfvs5x2g/cfYW7byMk+/L4u5S6+1J3f72Xfd9BfD/MrIbw7fmOjPNxkJnVuXuzu/91d06GmSVj7Je7+xZ3X0qoGXw8FjkXuM7dG+Lf6TW7s/8MHwNucffn3b0VuBw4zswmx9+hhlBDNHd/1d1XZ/x+e/N+FzUlhUHKzC4gfEP7E3BtxqpNhGaKMekF7v4VD/0K9xC+Saf91d1HeOhTOAA4DPhmXDeW0CyQ3keK8C11XFy3Ii5LWxbXAZxN+BBaZmaPm9lxu/nrvZnxuoWQgPoyCTgmJrhGM2skfKAcsI/iWZPxelsP89UAZrafmd1pZivNrAn4OVDXx377OsdpKzLWLwa+SKjNrI3HGtvLvm8HzjKzcuAs4Hl3Tx/rU4Raymtm9qyZnd5HjD2pI9QQl2Usy3z/x2bG3e317uh+fpoJTXPj4heRHxKaGteY2Y1mVhuL7u37XdSUFAYhM9uP0J76GeDvgHPN7AQAd99KaIc/a3f26e5rCN+uPxAXrSJ82KaPaYRmjJVx3QQzy/z7mRjX4e7PuvuZhCaW3wB3pQ+zOzHthhXA4zHBpadqd//cAMfzrbjPI9y9FriA0LSR1v14fZ3jHrdx99vd/Z1xO2fnLwSZ5eYTPlBPBT5KSBLpdYvc/XzC+bgW+KWZVWX/a7Ke8G18Usayzvef0HwzPmPdhN3Yd6bu56cKGE3X39kP3P3thC8zBwP/HJf39n5LFpQUBqcfAr9x90djlfkrwE3xWyFx/pNmdllMIJjZeGBKbzs0s9GEzsh5cdFdwPvN7GQzKwX+idDZ+X+EpLMV+IqZlZrZiYRkcqeZlZnZx8xseGwSaSI0e0D4hj3azIbvo/OQdh9wsJl9PMZTamZHmdmhAxxPDaETudHMxhE/pDKsAaZmzPd1jndhZtPN7D3xfd5OqKV09FQ2uh24FDiB0KeQ3s8FZlYfayaNcXGv+zGzisyJUBO9C7g6Xg49CfhHQs0o/Xt9wczGmdkIQud4f0q7Hackxn+Rmc2Mv/M3gafdfWl8f4+J521rPB8d/bzfko18d2po2r0J+CDhG9SIbssfBq7OmD8GeIDwT98IvAJcDYyO6y8k/LOkr7xZS2hz3i9jHx8idBhuBh4ndtzGdYfFZZtjmQ/F5WXAg4RmrCbgWeCdGdvdQmgCaKT3q4/+PWP+RPrpnI7lphOuWFoX9/8IoS9it+Kh547mkozyDWR04hM+CL+WcU6ei+dzLuFDviGj7JnA8nisL2dxjpeyc8f0EYS+ny2Ezvj7ejqHGeUnEj7A7++2/Ofx/W4mfAn4YC/bnxh//+7TQYS+q5/H870CuIKuq49KCDXZDYSrj75EqFlYL8dZ2sMx/j2u+yyh0zz9+46Py08mXHHUTNeVXtX9vd+a+p8snmARkZwws1OBH7v7pH4LS96p+UhE9ikzG2Zmp5lZSWxG+zfCRQ4yCKimIIOCmb0L+H1P6zxcPSUFwswqCU1hhxD6Pe4HvuDuTXkNTLKipCAiIp3UfCQiIp0G3SBbdXV1Pnny5HyHISIyqDz33HPr3b2+v3KDLilMnjyZOXPm5DsMEZFBxcyW9V9KzUciIpJBSUFERDopKYiISCclBRER6aSkICIinZQURESkk5KCiIh0KpqksGhZAw888hipjlT/hUVEilTRJIWNT/yE0544k23bmvMdiohIwSqapGBlYSDNli2N/ZQUESleRZMUkhU1AGzbqtF7RUR6U0RJIdQUdrQoKYiI9KZokkJpZS0AraopiIj0qmiSQtmw0HzUtn1LniMRESlcRZMUyqtiUtimpCAi0puiSQoVVcMB6FBSEBHpVdElBW9VUhAR6U3RJIXK6nRS2JrnSEREClfOkoKZ3WJma83slV7Wm5n9wMwWm9lLZnZkrmIBKCmroM2TsEN3NIuI9CaXNYXbgNl9rD8VmBani4Ef5TAWMKPFhmE7VFMQEelNzpKCuz8BbOyjyJnATz34KzDCzMbkKh6A7VZBol1JQUSkN/nsUxgHrMiYb4jLdmFmF5vZHDObs27duj0+4HYbRrJNSUFEpDf5TArWwzLvqaC73+jus9x9Vn19/R4fcEeiktKOlj3eXkRkqMtnUmgAJmTMjwdW5fKAO5LDKFNSEBHpVT6Twr3A38arkI4FNrv76lwesL2kirLUtlweQkRkUCvJ1Y7N7A7gRKDOzBqAfwNKAdz9x8ADwGnAYqAFuChXsaS1l1QxLKWagohIb3KWFNz9/H7WO/D5XB2/J6myaipdSUFEpDdFc0czgJfXUsU2OvScZhGRHhVVUqC8hlLroLlZ4x+JiPSkqJJCclgY/6i5aVOeIxERKUxFlRRKYlLY3qykICLSk+JKCpXppNCY50hERApTUSWFsuqRALSqpiAi0qOiSgoV8ZkK7S2b8xyJiEhhKqqkMKwm1BTatzXlORIRkcJUVEmhunYUAKltqimIiPSkqJJCeedzmlVTEBHpSVElBUuW0kI5iVbdvCYi0pOiSgoALVZJYoeSgohIT4ouKWxLlVKyeVm+wxARKUg5GyW1UG0rr6PMi+7XFhHJStHVFFpLhzOsQx3NIiI9Kbqk0F42nOqU+hRERHpSdEkhVTGSWt9CKuX5DkVEpOAUXVJg2CiqbTtNW7fmOxIRkYJTdEkhWRWGumjatD7PkYiIFJ6iSwol1WGoi+bGdXmORESk8BRdUiivrQNge5NqCiIi3RVdUhgWk0Jrk2oKIiLdFV1SqBmxHwDtWzfmORIRkcJTdEmhemQ9AB1b9fQ1EZHuii4pJIcNp50Etk1JQUSku5wmBTObbWYLzGyxmV3Ww/pJZvawmb1kZo+Z2fhcxhMPSjPVJFobc34oEZHBJmdJwcySwA3AqcAM4Hwzm9Gt2H8CP3X3I4CrgG/lKp5MW5M1lO1QUhAR6S6XNYWjgcXuvsTddwB3Amd2KzMDeDi+frSH9TnRkhxOeZseySki0l0uk8I4YEXGfENclulF4Oz4+kNAjZmN7r4jM7vYzOaY2Zx16/b+UtIdpbUMa9dIqSIi3eUyKVgPy7qPQvdl4N1m9gLwbmAl0L7LRu43uvssd59VX1+/14G1l4+gyjVSqohId7l82kwDMCFjfjywKrOAu68CzgIws2rgbHfPebtOqmIEw72Z9o4UJcmiuwBLRKRXufxEfBaYZmZTzKwMOA+4N7OAmdWZWTqGy4FbchhPl4qR1Ng2Nje3DMjhREQGi5wlBXdvBy4B/gC8Ctzl7vPM7CozOyMWOxFYYGYLgf2Bq3MVT6Zkdei22NKo8Y9ERDLl9GHF7v4A8EC3ZVdkvP4l8MtcxtCTkqqMkVInTR7ow4uIFKyibFCviIPi3fyH5/IciYhIYSnKpFAzKgyKd1bnLRIiIgJFmhRq68YC0Db2qDxHIiJSWIoyKZQPPwAAb16T50hERApLUSYFSsppsmqSLXrQjohIpuJMCsCW5CgqWnVJqohIpqJNCi1lo6hq25DvMERECkrRJoUdFfXUdjTi3n04JhGR4lW0SSFVVU8djWxp3WX8PRGRolW0SeGva5JU23bWb9RjOUVE0oo2KZz49sMB2LxuZZ4jEREpHEWbFIaNCjewbd2wqp+SIiLFo2iTQu3okBRaG1fnORIRkcJRtEmhpi48GbS9SXc1i4ikFW1SsKp6UhjWvDbfoYiIFIyiTQokS2iyWkq2a6gLEZG04k0KQHPJKCpadVeziEhaTp+8Vuga2mup7ND4RyIiaUVdUxhRP5Y6a6S1vSPfoYiIFISiTgrUjmU/NvFmY0u+IxERKQhFnRRKR02izDpYt3pZvkMRESkIRZ0UqvebAsCWN9/IcyQiIoWhqJPCiLFTAWhdvzS/gYiIFIiiTgrloyeHF5tX5DUOEZFCkdOkYGazzWyBmS02s8t6WD/RzB41sxfM7CUzOy2X8eyivJpGr6bpzSUDelgRkUKVs6RgZkngBuBUYAZwvpnN6Fbsa8Bd7v424Dzgv3IVT2+2VIxhYlI3sImIQBZJwcwONLPy+PpEM7vUzEZkse+jgcXuvsTddwB3Amd2K+NAbXw9HBjwcay3V45lVNsa2jpSA31oEZGCk01N4VdAh5kdBPw3MAW4PYvtxgGZjfUNcVmmK4ELzKwBeAD4h552ZGYXm9kcM5uzbt2+HavIR0xgnK1n5UbdqyAikk1SSLl7O/Ah4Pvu/iVgTBbbWQ/LvNv8+cBt7j4eOA34mZntEpO73+jus9x9Vn19fRaHzl5F3WSqrJWG1XoCm4hINkmhzczOBz4B3BeXlWaxXQMwIWN+PLs2D30KuAvA3Z8CKoC6LPa9zwwfEy5L3bRKnc0iItkkhYuA44Cr3f0NM5sC/DyL7Z4FppnZFDMrI3Qk39utzHLgZAAzO5SQFAZ0LOva/UNS+POc5wfysCIiBanfUVLdfT5wKYCZjQRq3P2aLLZrN7NLgD8ASeAWd59nZlcBc9z9XuCfgJvM7EuEpqUL3b17E1NO2YiJAMys2TKQhxURKUj9JgUzeww4I5adC6wzs8fd/R/729bdHyB0IGcuuyLj9XzgHbsZ875VOYrtlFG+cX5ewxARKQTZNB8Nd/cm4CzgVnd/O/De3IY1gMxorJxCnW+iIzWglRQRkYKTTVIoMbMxwLl0dTQPKa3DpzKZ1axq3JbvUERE8iqbpHAVoV/gdXd/1symAotyG9bAStRPY7ytZ9najfkORUQkr/pNCu5+t7sf4e6fi/NL3P3s3Ic2cGrHH0rCnHXLXs13KCIieZXNMBfjzeweM1trZmvM7FdmNn4gghsoteMPBWDrSiUFESlu2TQf3Uq4v2AsYZiK38VlQ4aNnhZerF+c30BERPIsm6RQ7+63unt7nG4D9u1YE/lWXk1TaR3VW98gpSuQRKSIZZMU1pvZBWaWjNMFwJAba3pb7VQm+ioaNukKJBEpXtkkhU8SLkd9E1gNnEMY+mJISdRPY6qt5rXVm/MdiohI3mRz9dFydz/D3evdfT93/yDhRrYhpXbcDEbYVpat0KM5RaR47emT1/od4mKwKR8bHgrX0vBSniMREcmfPU0KPT0rYXA74AgAyte/kudARETyZ0+TwtC7RKeqjjWMZv+WhWxtbc93NCIiedFrUjCzLWbW1MO0hXDPwpCTHDeTw20pL65ozHcoIiJ50WtScPcad6/tYapx936H3B6MqpteZ6qt4sUl3R8QJyJSHPa0+WhIqnj/N0mas2GJnsImIsVJSSHTmLcCYG++rDubRaQoKSlkqh1Ha+kIpra/zpL1zfmORkRkwCkpZDKjw0p4a2IJzy3blO9oREQGXDZDZ/d0FdKKOJz21IEIciANO+7THGLL+a8H1a8gIsUnm6uIvgusAm4n3LR2HnAAsAC4BTgxV8Hlg006HjPnrb6QVMpJJIbefXoiIr3Jpvlotrv/xN23uHuTu98InObu/wuMzHF8A2/8UaSshOk7Xmb+6qZ8RyMiMqCySQopMzvXzBJxOjdj3dC7RKesko4D3srRidf47M+fy3c0IiIDKpuk8DHg48DaOH0cuMDMhgGX5DC2vCmdcjwzE0uYUKN+eBEpLtkMnb3E3T/g7nVx+oC7L3b3be7+5762NbPZZrbAzBab2WU9rP+emc2N00IzK4zxJSYeTyntpBrm0NiyI9/RiIgMmGyuPhofrzRaa2ZrzOxXZjY+i+2SwA3AqcAM4Hwzm5FZxt2/5O4z3X0mcD3w6z37NfaxicfiwPGJV3ho/pp8RyMiMmCyaR+5FbiXMAjeOOB3cVl/jgYWx5rGDuBO4Mw+yp8P3JHFfnOvchRMOIbZpS9y74saB0lEikc2SaHe3W919/Y43QbUZ7HdOCDzMWYNcdkuzGwSMAV4JIv9Dgg7+H1M9yUsWLSI9c2t+Q5HRGRAZJMU1pvZBWaWjNMFwIYstuvpAv/erlY6D/ilu3f0uCOzi81sjpnNWbduXRaH3gfm/w6Ak5JzeeDl1QNzTBGRPMsmKXwSOBd4E1gNnANclMV2DcCEjPnxhJvgenIefTQdufuN7j7L3WfV12dTSdkHLn4Uhk/gjGEv85sXVg7MMUVE8iybq4+Wu/sZ7l7v7vu5+weBs7LY97PANDObYmZlhA/+e7sXMrPphJvgntrN2HPLDDraObp9DvOWr2X+Kt3IJiJD355eiP+P/RVw93bCfQx/AF4F7nL3eWZ2lZmdkVH0fOBOdy+8G+HO+jGltDO7dC4/fWppvqMREcm5PX2CWlYDArn7A8AD3ZZd0W3+yj2MIfcmvwuq9+cziRc4Z+6xXHbqIYyoLMt3VCIiObOnNYXC+1afC4kkWIIZTU9S2tbM6df3ea+eiMig12tS6GXI7CYz20K4Z6E4nPszEqT44viFbG5pY+NW3eEsIkNXr0nB3WvcvbaHqcbd97TZafAZPwtKKjh/881saW3ntOueyHdEIiI5oxHf+mMGJ32VyraNvKN6FWuaWlm+oSXfUYmI5ISSQjaO/DhYgpvLvktFaZJr//BaviMSEckJJYVsDBsJsz7JsNb1fOm44dz/0mr+pIHyRGQIUlLI1jGfg44dfOrVT1NZluRzv3iOTep0FpEhRkkhW3UHQWU9yeZV3POJ6bR3OCd95zFSqeK4OldEioOSwu646AFwZ/qSW/m3D8ygsaWNE779aL6jEhHZZ5QUdkf9wVBVB/93PZ+YkWB0VRkNjdu47yU9c0FEhgYlhd31mVAzsJtO4i+XvYea8hL+4fYXeHzhAA3pLSKSQ0oKu2vEBDjpq9CygYplj/Lny97DsLIkF97yDL96riHf0YmI7BUlhT1x/D9AyTC443yG08xTl51MTUUJ/3T3ixx/zcPqfBaRQUtJYU+UlIdO5442uH4Ww4eV8Nz/O4X66nJWNW7nE7c+w7oteoSniAw+Sgp7atyRMGIitKyHl++mNJngma+ezJTRlTy5aD3HX/Mwj7ymG9xEZHBRUtgbl74A5bVwz2dh3ULMjEf/+ST+8MUTKEkk+ORtczjyqj/y5ubt+Y5URCQrSgp7I5GEv38KLAE/OQG2NQIw/YAa5v7bKXz5bw5mY0sbx37rYf77z2/Q3pHKc8AiIn2zQnwKZl9mzZrlc+bMyXcYO1v2f3DraaHW8E+vQVll56rlG1r4wA//zOZtbZjBhJHDuPeSd+oJbiIyoMzsOXef1V851RT2hUnHw1k3Qetm+O6h0NbVXDRxdCVzrziFn3z87VSVlbB84zZmXvUQR/37Q7yycnMegxYR2ZVqCvvS9bNgwyKY9jdw7k+hdNguRU7/wZMsWLOFto5w3hMGk0dX8cAX3kVFaXKgIxaRIpFtTUFJYV/7wdth42Ior4EvvASVo3ostnlbG2dc/2eWbQwP7DFg/9oKfvjRt3HkxJEkEjaAQYvIUKekkE/z7oG7Lwr3M1z8OOx3SK9F3Z1Tr3uSxWubac+46a2+ppwrTp/BsVNHU19TPhBRi8gQpqSQb8v/CredDp6Cj/wcDjmt302atrfxoRv+wrINLTsliIRBfXU5Xz/zMI6ZMpqRVeqkFpHdo6RQCDY3wA3HwI7mMF7Su74Miez69ts7Ury8cjOX3vECKzZt22V9ScK47NRDmDG2lsPGDGd4Zem+jl5EhhAlhULRtg2+/xbYug4OOgXe/x0YOWn3d9OR4qWGRr5w51waekgSACMrS7ng2EkcvH8N0w+oYUpdFaVJXWAmIgWSFMxsNnAdkARudvdreihzLnAl4MCL7v7RvvY56JICgDs8cyM8eFmYP/kKOO4SSO7dt/sNza3MW9XEV+95mdWbt5Nyp/tYfAmDkZVlXHj8ZCbXVTFxVCWTRlcyfFgpZurMFikWeU8KZpYEFgKnAA3As8D57j4/o8w04C7gPe6+ycz2c/e1fe13UCaFtMYV8c7njVB/KJz+PZh03D49RGt7B6+v3crCNVu49sHXWLullZQ7vb3NJQmjvqacz590EAfUVlBfU059TTmjq8soL9ElsiJDRSEkheOAK939fXH+cgB3/1ZGmW8DC9395mz3O6iTQtprD8DdF0JHK7ztAnjvVVA1OqeH3NrazopNLSzf0MK/3/8qKza24ISaRH8jfScTxohhpZw5c1xn0qivKaeuuiwkkKpykrqEVqSgFUJSOAeY7e6fjvMfB45x90syyvyGUJt4B6GJ6Up3f7CHfV0MXAwwceLEty9btiwnMQ+oHVvh8W/DX64LYyid/j1460chWTLgoaRSzpot21nb1Mq6La2sa27lhkcWs7JxGw4kDVLQa20Dwn0WWFeZkpgkRleVcfbbx1NTUUp1RQm1FSVUl5eE+fISairCVF1eQon6P0RyphCSwoeB93VLCke7+z9klLkPaAPOBcYDTwKHu3tjb/sdEjWFTGvmwy3vg9am8OCeM34Ah58dEkUB2trazvrmkDzWN7dyze9fY1VMHu50XkqbMDqX7Q4jbAehhpJONnVVZZx71MSdk0p8XVtRQnVFWFZZmtSNfyI9yDYp5PJraQMwIWN+PND9CfcNwF/dvQ14w8wWANMI/Q/FYf8ZcNlyeO1++PVnwnTvpXDG9XDYh/JSc+hLVXkJVeUlTBpdBcDsw8f0u427s70txZbWNrZsb6d5e3v42drGtx9cwBvrtwJQkgwf5u0dHhNK+JlKwZtNrfzg4UW7HW86P5gZHSnHuh0HoKwk1FB2tIdRbMtLw/zEUZVccfphlJUkwpQMP8u7zZeVJChJmDruZUjIZU2hhNA0dDKwkvBB/1F3n5dRZjah8/kTZlYHvADMdPcNve13yNUUMqVS8NrvwvMZ2logWQYnXg5vv7DX4TKKSSrlNO/YOak0xSTz5btfZEd7itKk4XR94KdrDZ5xZVZmbSSXzMKx0sdNWljYERekm9g655NdSSUdf2lJAqMrYU0aXcmn3zWVZMJImmEWalTJhJGwMCUTZLw2ErFswsL5CGXpWp9RNpkICTQZ5zv3b2E/CUu/3nn79D6VGAtX3puPYhCnAd8n9Bfc4u5Xm9lVwBx3v9fCX9B3gNlAB3C1u9/Z1z6HdFJIS6Vg0R/hnoth++bwvIYj/xaO+Szsd2i+oxtS2jtStLan2NGeYkdH+Nl9fkd7iqvum8fyjS20toUP5+61i3QyCvsMr9Kd7+kmtWQidLrE1Z21mIFOVvmSmS7Sv2c6h3gv56Szpged5y3ZPZl2m09mbNTRsXPCbe9nPpsy7R1OSdKoqy5nfXMr7R1OadIYXV3OhubwGN70gJelcZvM+brqrmFrzjpy/C7n6dfPN4DB2UeOx4BfPd/Q+XjfafvXcP+l79plm2wURFLIhaJICpnWzIeffRCa46M9p54Ix3wOpp1SsP0Osm90pJwd7SnaUylSKehwD/eipJwOdzpS4VLjjjjv7nSkwnwqlu16HZen4mt3vnn/fBataQagLDaZpZNeeWkCHFrbd06C0JUIy0oSONCWkRih6wMwmw/ezA94p+sDvr37BzxdH/q9JY7O5JrexDOST9eizvnB9ckXTB5dyWP/fNIebaukMNRs3QDP3waPXQMdO0LT0nGfh7eeD/XT8x2diAwAd9/jJjolhaGqoy10St/3Rdi2KSwrq4b3XhmuWlLfg4j0QEmhGDSvhZfvhj9+LYzGisGhp4f7HaadstfDaIighgYiAAAPp0lEQVTI0KGkUGzefBnm3hHGWEq1AQbV+8PZN8Gkd6j/QaTIKSkUq442WPww/Pbz0LI+LEuUhquXDnk/TH5nePiPiBQVJQWBHS3h0tZ598D834RllgzJYfqp4VnSVXX5jVFEBkQh3NEs+VZWCYd9MExt22DJ47Dw9zD3dnj13lCmvBbe9Y8hQew3I+N6PhEpRqopFCN3WD0XFjwIT10fBucDwELN4ZRvhPshavsfwkJEBgc1H0n2Nq+EJY/CQ1fGfoj0XUAJqD4APvD90FldXp3PKEVkLygpyJ5JpWDNKyFJ/Onr4B1d6ywJtePgw7fC2LfpiiaRQURJQfaNtu2w/Cn47SXQ1LDzukQJjJgI590O9YeoP0KkgCkpSG5s3QBvPAYPXh5unsscTSYRaxJn3gDjjoSyqjwGKiKZlBQk99yhcRm88SQ8fFXoj/BU13pLhBvoTvlGSBKjpqo2IZInSgqSHy0boWEOrHganr0pDP2dyZJQMwZOvTb0S9SOVaIQGQBKClIYOtph3auw8nl49GrYum7n2gQWahQ1Y+CUr8MBR8DoA9WJLbKPKSlI4WrbBm++AqtegCf+A7au3bVMWXUY9fWAt4Rp/8OgvGbgYxUZIpQUZHBp3wHrF4SB/R7+RniokKfY6VEoJRVh9Nf9Y6I44HAYPkHNTyJZUFKQwc8dmlaGWsXvvxJee6pb8xNhqI6ZHwtJYv/DwyNLNeifyE409pEMfmYwfHyYps/uWt7aDGvnw5svweP/Efopnv5Rt20TUDkajr+0qwlKg/+J9Es1BRkaUh2wcUloflrzCvzlOki171ouURITxeGhGUqd2lIk1HwkAuFmuzUvh2Txl+ugZcOuzU9l1fCWc2LT04xwd3bV6PzEK5IjSgoivWlvhXWvhb6KN18OQ4m3drufAgt9FUd8OCSJ9FRdn5eQRfaWkoLI7nCHzQ2wbkFIGOtehXm/hR1buhW0cGnsW86B+kOhfnpMFvvpKigpaEoKIvuCOzStiokiJox5v4bWbskiUQLjjwpJou5gGH1QmEZMhGRpfmIXyVAQScHMZgPXAUngZne/ptv6C4H/AFbGRT9095v72qeSghQE93AvxbrXYO1rXUmj4ZldO7hLhsGBJ4VkUT8d6qZD3TSoqM1P7FKU8n5JqpklgRuAU4AG4Fkzu9fd53cr+r/ufkmu4hDJCTOoOSBMU0/ceV3LRtiwuGtavwgW/wkWPLBzuWQZTDwuJIrRB8GoA8PVUMMnQFJXi0t+5PIv72hgsbsvATCzO4Ezge5JQWRoqRwFlUfDhKN3Xt7RDpuWhju31y2A9QvDz2du3HUfJcNg6rtjopjalTBqx0MiMSC/hhSnXCaFccCKjPkG4Jgeyp1tZicAC4EvufuK7gXM7GLgYoCJEyfmIFSRAZAsgbqDwnTI+7uWp5uiNrwOG18PP1/4GSz6466XzwKUVsLUk0KSGH1gqGXUHQxV9erslr2Wy6TQ019n9w6M3wF3uHurmX0W+B/gPbts5H4jcCOEPoV9HahIXmU2RU1+R1h2ytfDz1QKtqzuShYbX4cNS2Dh73tOGGU1IeGMmgIjJ8dpiq6OkqzlMik0ABMy5scDqzILuPuGjNmbgGtzGI/I4JNIwPBxYZpyws7rUh1hPKh0v8X6hWGa92vo2LHrvkqr4KCTw8OOMqeaMWqSkk65TArPAtPMbArh6qLzgI9mFjCzMe6+Os6eAbyaw3hEhpZEMlzyOmIiHNitgt3eCo0rYNMbsPGN+HMJLHoI2rftuq/SytBhPmpqqGWkE0bteHV6F5mcvdvu3m5mlwB/IFySeou7zzOzq4A57n4vcKmZnQG0AxuBC3MVj0hRKSnv6r/oLl3D2LgkY4pJY+GDuzZLWQIOem9XohgZk8aICRqNdgjSzWsi0iWVguY3uyWMJbDoT9C2tYcN4h3eh36gq/8iXdsYNlL9GAUk7/cpiMgglEiE52bXjoXJ79x5nXsYpjydKDYtg8Zl8Nr9YfyoXa4jIQw2OO2UnZPFyCnqxyhgSgoikh2zcBVT9X4w8dhd17dtC/dhpJuiNr0BL/8S5t3T+z6HjYQjPhISxcjJIWkMnwBllbn6LaQfaj4SkdzraIPNK2Kn99KQMF64HbZv6vnS2rTK0eGpep2X2E7RHd97qCDGPsoFJQWRIcYdtq4PiWLTUmhcDs/cFJqq8F6ShkHFcDj87J2vmBo5GUqHDWz8g4SSgogMDakUbFmVcWntG/D8T6FlfS8bWLhiathIeNvHQs1i+ITwWNcRE0IyKUJKCiJSHFo27nwvxjM3ZTxhr5fPN0t01TSGj985cdQcMCQf0aqkICKSSsHWteEBSo3Lw8/NK0IH+PbGvvszymth+mmhdtE9cQzCjnAlBRGRbLRuickiJo4nvxvu1XDvu7ZRVh3uAh8xMSNpjA/zlaML7h4N3acgIpKN8hrY79AwARz1qZ3Xd7SHPo3NDWHokM0rujrCFzzQd23DklBVB0d9eufEUTsOSspy9zvtBdUURET2hjts2xSSReOKkDz+cl1otvJUWN9r30Yy3v+xP7zzSzntEFfzkYhIoWjbHsabSvdrPH4tbHkzJo0+mqggPP+7sh6O/Ts46BQ44PA9CkFJQURksEh3iKebpzavgIe/Aam2cKVUuolq1EFw6XN7dAj1KYiIDBaJRNeDliYcFZa94ws7l9neNCCXyiopiIgMBhW1A3IYDVMoIiKdlBRERKSTkoKIiHRSUhARkU5KCiIi0klJQUREOikpiIhIJyUFERHpNOiGuTCzdcCyPdy8DujtcU2FotBjLPT4QDHuC4UeHxR+jIUW3yR3r++v0KBLCnvDzOZkM/ZHPhV6jIUeHyjGfaHQ44PCj7HQ4+uNmo9ERKSTkoKIiHQqtqRwY74DyEKhx1jo8YFi3BcKPT4o/BgLPb4eFVWfgoiI9K3YagoiItIHJQUREelUNEnBzGab2QIzW2xml+U7HgAzW2pmL5vZXDObE5eNMrOHzGxR/DlygGO6xczWmtkrGct6jMmCH8Rz+pKZHZnHGK80s5XxXM41s9My1l0eY1xgZu8bgPgmmNmjZvaqmc0zsy/E5QVxHvuIr5DOYYWZPWNmL8YYvx6XTzGzp+M5/F8zK4vLy+P84rh+ch5jvM3M3sg4jzPj8rz8v+w2dx/yE5AEXgemAmXAi8CMAohrKVDXbdm3gcvi68uAawc4phOAI4FX+osJOA34PWDAscDTeYzxSuDLPZSdEd/vcmBK/DtI5ji+McCR8XUNsDDGURDnsY/4CukcGlAdX5cCT8dzcxdwXlz+Y+Bz8fXfAz+Or88D/ncA/g57i/E24Jweyufl/2V3p2KpKRwNLHb3Je6+A7gTODPPMfXmTOB/4uv/AT44kAd39yeAjVnGdCbwUw/+CowwszF5irE3ZwJ3unuru78BLCb8PeSMu6929+fj6y3Aq8A4CuQ89hFfb/JxDt3dm+NsaZwceA/wy7i8+zlMn9tfAiebmeUpxt7k5f9ldxVLUhgHrMiYb6Dvf4KB4sAfzew5M7s4Ltvf3VdD+OcF9stbdF16i6nQzuslsVp+S0azW15jjM0YbyN8iyy489gtPiigc2hmSTObC6wFHiLUUBrdvb2HODpjjOs3A6MHOkZ3T5/Hq+N5/J6ZlXePsYf4C0axJIWevjEUwrW473D3I4FTgc+b2Qn5Dmg3FdJ5/RFwIDATWA18Jy7PW4xmVg38Cviiuzf1VbSHZTmPsYf4CuocunuHu88ExhNqJof2EUdBxGhmhwOXA4cARwGjgH/JZ4y7q1iSQgMwIWN+PLAqT7F0cvdV8eda4B7CH/6adJUy/lybvwg79RZTwZxXd18T/0FTwE10NW/kJUYzKyV84P7C3X8dFxfMeewpvkI7h2nu3gg8RmiHH2FmJT3E0RljXD+c7JsY92WMs2PznLt7K3ArBXIes1UsSeFZYFq8cqGM0BF1bz4DMrMqM6tJvwb+BnglxvWJWOwTwG/zE+FOeovpXuBv41UVxwKb080jA61b2+yHCOcSQoznxatTpgDTgGdyHIsB/w286u7fzVhVEOext/gK7BzWm9mI+HoY8F5C38ejwDmxWPdzmD635wCPeOzdHeAYX8tI/Ebo88g8jwXx/9KnfPd0D9RE6PlfSGiX/GoBxDOVcEXHi8C8dEyEdtCHgUXx56gBjusOQtNBG+Gbzad6i4lQHb4hntOXgVl5jPFnMYaXCP98YzLKfzXGuAA4dQDieyehWeAlYG6cTiuU89hHfIV0Do8AXoixvAJcEZdPJSSkxcDdQHlcXhHnF8f1U/MY4yPxPL4C/JyuK5Ty8v+yu5OGuRARkU7F0nwkIiJZUFIQEZFOSgoiItJJSUFERDopKYiISCclBRER6aSkIJIFM5vZbSjpM2wfDcFuZl80s8p9sS+RvaX7FESyYGYXEm42uiQH+14a971+N7ZJunvHvo5FRDUFGVLMbLKFh8fcFB988sc4BEFPZQ80swfjKLVPmtkhcfmHzeyV+PCUJ+LQKFcBH4kPTfmImV1oZj+M5W8zsx9ZeHDNEjN7dxxl9FUzuy3jeD8yszm28wNZLgXGAo+a2aNx2fkWHr70ipldm7F9s5ldZWZPA8eZ2TVmNj+OxvmfuTmjUnTyfUu1Jk37cgImA+3AzDh/F3BBL2UfBqbF18cQxsuBMATBuPh6RPx5IfDDjG075wkPVbmTMIzBmUAT8BbCl67nMmJJD2uRJAyedkScX0p82BIhQSwH6oESwpAJH4zrHDg3vS/CkBOWGacmTXs7qaYgQ9Eb7j43vn6OkCh2EoeNPh64O46H/xPCE8kA/gLcZmafIXyAZ+N37u6EhLLG3V/2MNrovIzjn2tmzxPGyzmM8ESz7o4CHnP3dR6eC/ALwpPmADoII5tCSDzbgZvN7CygJcs4RfpU0n8RkUGnNeN1B9BT81GC8MCWmd1XuPtnzewY4P1A5zN2szxmqtvxU0BJHF30y8BR7r4pNitV9LCfvp4Wtt1jP4K7t5vZ0cDJhFF/LyE8lUxkr6imIEXJw0Nl3jCzD0PnQ9XfGl8f6O5Pu/sVwHrCGPhbCM8z3lO1wFZgs5ntT3iwUlrmvp8G3m1mdWaWBM4HHu++s1jTGe7uDwBfJDwYR2SvqaYgxexjwI/M7GuE5+veSRjK/D/MbBrhW/vDcdly4LLY1PSt3T2Qu79oZi8QmpOWEJqo0m4Efm9mq939JDO7nPDcAAMecPeenqlRA/zWzCpiuS/tbkwiPdElqSIi0knNRyIi0knNRzLkmdkNwDu6Lb7O3W/NRzwihUzNRyIi0knNRyIi0klJQUREOikpiIhIJyUFERHp9P8BdGFjB2OqPC8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 画出多类分的logloss 结果图\n",
    "test_means = cvresult_round1['test-mlogloss-mean']\n",
    "test_stds = cvresult_round1['test-mlogloss-std'] \n",
    "        \n",
    "train_means = cvresult_round1['train-mlogloss-mean']\n",
    "train_stds = cvresult_round1['train-mlogloss-std'] \n",
    "\n",
    "x_axis = range(0, cvresult_round1.shape[0])\n",
    "        \n",
    "plt.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "plt.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "plt.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "plt.xlabel( 'n_estimators' )\n",
    "plt.ylabel( 'Log Loss' )\n",
    "plt.savefig( 'first_round_nestimators.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "376\n"
     ]
    }
   ],
   "source": [
    "n_estimators = cvresult_round1.shape[0]\n",
    "print(n_estimators)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb1.n_estimators = n_estimators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "first term of train logloss 0.4814292748474865\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import log_loss\n",
    "xgb1.fit(x_train,y_train,eval_metric='mlogloss')\n",
    "\n",
    "train_predprob = xgb1.predict_proba(x_train)\n",
    "logloss = log_loss(y_train,train_predprob)\n",
    "print('first term of train logloss',logloss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 对树的最大深度进行调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "range(1, 10, 2)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_depth_list = range(1,10,2)\n",
    "param_List_1 = {'max_depth':max_depth_list}\n",
    "max_depth_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
       "       colsample_bytree=0.8, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=5, min_child_weight=1, missing=None, n_estimators=376,\n",
       "       n_jobs=1, nthread=None, objective='multi:softprob', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=3, silent=True,\n",
       "       subsample=0.8),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'max_depth': range(1, 10, 2)}, pre_dispatch='2*n_jobs',\n",
       "       refit=True, return_train_score='warn', scoring='neg_log_loss',\n",
       "       verbose=0)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid1 = GridSearchCV(estimator=xgb1,param_grid=param_List_1, scoring='neg_log_loss',n_jobs=-1,cv=3)\n",
    "grid1.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jack/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.63610, std: 0.00285, params: {'max_depth': 1},\n",
       "  mean: -0.59479, std: 0.00250, params: {'max_depth': 3},\n",
       "  mean: -0.58532, std: 0.00232, params: {'max_depth': 5},\n",
       "  mean: -0.59477, std: 0.00164, params: {'max_depth': 7},\n",
       "  mean: -0.62644, std: 0.00251, params: {'max_depth': 9}],\n",
       " {'max_depth': 5},\n",
       " -0.5853164265967271)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid1.grid_scores_, grid1.best_params_, grid1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=4, error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
       "       colsample_bytree=0.8, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=5, min_child_weight=1, missing=None, n_estimators=376,\n",
       "       n_jobs=1, nthread=None, objective='multi:softprob', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=3, silent=True,\n",
       "       subsample=0.8),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'max_depth': [4, 5, 6]}, pre_dispatch='2*n_jobs',\n",
       "       refit=True, return_train_score='warn', scoring='neg_log_loss',\n",
       "       verbose=0)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb1.max_depth = grid1.best_params_['max_depth']\n",
    "param_List_2 = {'max_depth':[4,5,6]}\n",
    "grid2 = GridSearchCV(estimator = xgb1, param_grid=param_List_2,scoring='neg_log_loss',n_jobs=-1,cv=4)\n",
    "grid2.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jack/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58795, std: 0.00243, params: {'max_depth': 4},\n",
       "  mean: -0.58520, std: 0.00247, params: {'max_depth': 5},\n",
       "  mean: -0.58646, std: 0.00308, params: {'max_depth': 6}],\n",
       " {'max_depth': 5},\n",
       " -0.5851957017423537)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid2.grid_scores_,grid2.best_params_,grid2.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 对min_children_weight进行调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb1.max_depth = grid2.best_params_['max_depth']\n",
    "\n",
    "min_child_weight_list = range(1,10,2)\n",
    "param_List_3 = {'min_child_weight':min_child_weight_list}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid3 = GridSearchCV(estimator=xgb1,param_grid = param_List_3,scoring='neg_log_loss',n_jobs=-1,cv=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=4, error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
       "       colsample_bytree=0.8, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=5, min_child_weight=1, missing=None, n_estimators=376,\n",
       "       n_jobs=1, nthread=None, objective='multi:softprob', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=3, silent=True,\n",
       "       subsample=0.8),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'min_child_weight': range(1, 10, 2)},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid3.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jack/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58520, std: 0.00247, params: {'min_child_weight': 1},\n",
       "  mean: -0.58499, std: 0.00212, params: {'min_child_weight': 3},\n",
       "  mean: -0.58554, std: 0.00282, params: {'min_child_weight': 5},\n",
       "  mean: -0.58526, std: 0.00208, params: {'min_child_weight': 7},\n",
       "  mean: -0.58513, std: 0.00210, params: {'min_child_weight': 9}],\n",
       " {'min_child_weight': 3},\n",
       " -0.5849868399607913)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid3.grid_scores_, grid3.best_params_,grid3.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb1.min_child_weight = grid3.best_params_['min_child_weight']\n",
    "\n",
    "min_child_weight_list1=[2,3,4]\n",
    "param_List_4 = {'min_child_weight':min_child_weight_list1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jack/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58473, std: 0.00269, params: {'min_child_weight': 2},\n",
       "  mean: -0.58499, std: 0.00212, params: {'min_child_weight': 3},\n",
       "  mean: -0.58521, std: 0.00256, params: {'min_child_weight': 4}],\n",
       " {'min_child_weight': 2},\n",
       " -0.5847328595355582)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid4 = GridSearchCV(estimator=xgb1,param_grid=param_List_4,scoring='neg_log_loss',n_jobs=-1,cv=4)\n",
    "grid4.fit(x_train,y_train)\n",
    "grid4.grid_scores_,grid4.best_params_,grid4.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb1.min_child_weight = grid4.best_params_['min_child_weight']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 对正则参数进行调优"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## alpha调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jack/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58507, std: 0.00298, params: {'reg_alpha': 0.001},\n",
       "  mean: -0.58484, std: 0.00218, params: {'reg_alpha': 0.01},\n",
       "  mean: -0.58511, std: 0.00220, params: {'reg_alpha': 0.1},\n",
       "  mean: -0.58452, std: 0.00232, params: {'reg_alpha': 1},\n",
       "  mean: -0.58711, std: 0.00338, params: {'reg_alpha': 10},\n",
       "  mean: -0.61804, std: 0.00462, params: {'reg_alpha': 100},\n",
       "  mean: -0.70976, std: 0.00213, params: {'reg_alpha': 1000}],\n",
       " {'reg_alpha': 1},\n",
       " -0.5845163436904193)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg_alpha_List1 = [0.001,0.01,0.1,1,10,100,1000]\n",
    "param_List_5 = {'reg_alpha':reg_alpha_List1}\n",
    "\n",
    "grid5 = GridSearchCV(estimator=xgb1,param_grid=param_List_5,scoring='neg_log_loss',n_jobs=-1,cv=4)\n",
    "grid5.fit(x_train,y_train)\n",
    "grid5.grid_scores_,grid5.best_params_,grid5.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb1.reg_alpha = grid5.best_params_['reg_alpha']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## lambda调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jack/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58474, std: 0.00252, params: {'reg_lambda': 0.001},\n",
       "  mean: -0.58465, std: 0.00236, params: {'reg_lambda': 0.01},\n",
       "  mean: -0.58409, std: 0.00286, params: {'reg_lambda': 0.1},\n",
       "  mean: -0.58452, std: 0.00232, params: {'reg_lambda': 1},\n",
       "  mean: -0.58471, std: 0.00248, params: {'reg_lambda': 10},\n",
       "  mean: -0.58723, std: 0.00318, params: {'reg_lambda': 100},\n",
       "  mean: -0.59841, std: 0.00360, params: {'reg_lambda': 1000}],\n",
       " {'reg_lambda': 0.1},\n",
       " -0.5840895687168449)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg_lambda_List1 = [0.001,0.01,0.1,1,10,100,1000]\n",
    "param_List_6 = {'reg_lambda':reg_lambda_List1}\n",
    "\n",
    "grid6 = GridSearchCV(estimator=xgb1,param_grid=param_List_6,scoring='neg_log_loss',n_jobs=-1,cv=4)\n",
    "grid6.fit(x_train,y_train)\n",
    "grid6.grid_scores_,grid6.best_params_,grid6.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb1.reg_lambda = grid6.best_params_['reg_lambda']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 行列重采样参数调整"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## subsample调整"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "subsample_List1 = [0.3,0.4,0.5,0.6,0.7,0.8,0.9,1]\n",
    "param_List_7 = {'subsample':subsample_List1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jack/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.59186, std: 0.00273, params: {'subsample': 0.3},\n",
       "  mean: -0.58896, std: 0.00317, params: {'subsample': 0.4},\n",
       "  mean: -0.58730, std: 0.00238, params: {'subsample': 0.5},\n",
       "  mean: -0.58587, std: 0.00213, params: {'subsample': 0.6},\n",
       "  mean: -0.58472, std: 0.00304, params: {'subsample': 0.7},\n",
       "  mean: -0.58409, std: 0.00286, params: {'subsample': 0.8},\n",
       "  mean: -0.58361, std: 0.00221, params: {'subsample': 0.9},\n",
       "  mean: -0.58447, std: 0.00217, params: {'subsample': 1}],\n",
       " {'subsample': 0.9},\n",
       " -0.5836121779706169)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid7 = GridSearchCV(estimator=xgb1,param_grid=param_List_7,scoring='neg_log_loss',cv=4)\n",
    "grid7.fit(x_train,y_train)\n",
    "grid7.grid_scores_,grid7.best_params_,grid7.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb1.subsample = grid7.best_params_['subsample']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## colsample_bytree调整"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jack/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58379, std: 0.00216, params: {'colsample_bytree': 0.7},\n",
       "  mean: -0.58361, std: 0.00221, params: {'colsample_bytree': 0.8},\n",
       "  mean: -0.58415, std: 0.00191, params: {'colsample_bytree': 0.9},\n",
       "  mean: -0.58440, std: 0.00272, params: {'colsample_bytree': 1}],\n",
       " {'colsample_bytree': 0.8},\n",
       " -0.5836121779706169)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "colsample_bytree_list1 = [0.7,0.8,0.9,1]\n",
    "param_List_8 = {'colsample_bytree':colsample_bytree_list1}\n",
    "\n",
    "# GridSearchCV调优\n",
    "grid8 = GridSearchCV(estimator =xgb1,\n",
    "                       param_grid = param_List_8, \n",
    "                       scoring='neg_log_loss',\n",
    "                       n_jobs=-1, \n",
    "                       cv=4)\n",
    "grid8.fit(x_train,y_train)\n",
    "# 输出调优结果\n",
    "grid8.grid_scores_, grid8.best_params_, grid8.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb1.colsample_bytree = grid8.best_params_['colsample_bytree']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max_depth best is 5\n",
      "min_child_weight best is 2\n",
      "reg_alpha best is 1\n",
      "reg_lambda best is 0.1\n",
      "subsample best is 0.9\n",
      "colsample_bytree best is 0.8\n"
     ]
    }
   ],
   "source": [
    "print('max_depth best is',xgb1.max_depth)\n",
    "print('min_child_weight best is',xgb1.min_child_weight)\n",
    "print('reg_alpha best is',xgb1.reg_alpha)\n",
    "print('reg_lambda best is',xgb1.reg_lambda)\n",
    "print('subsample best is',xgb1.subsample)\n",
    "print('colsample_bytree best is',xgb1.colsample_bytree)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 重新调整弱学习器数目"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train-mlogloss-mean</th>\n",
       "      <th>train-mlogloss-std</th>\n",
       "      <th>test-mlogloss-mean</th>\n",
       "      <th>test-mlogloss-std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.092398</td>\n",
       "      <td>0.000013</td>\n",
       "      <td>1.092467</td>\n",
       "      <td>0.000018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.086355</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>1.086486</td>\n",
       "      <td>0.000043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.080416</td>\n",
       "      <td>0.000032</td>\n",
       "      <td>1.080607</td>\n",
       "      <td>0.000091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.074607</td>\n",
       "      <td>0.000057</td>\n",
       "      <td>1.074855</td>\n",
       "      <td>0.000055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.068885</td>\n",
       "      <td>0.000094</td>\n",
       "      <td>1.069194</td>\n",
       "      <td>0.000052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.063243</td>\n",
       "      <td>0.000013</td>\n",
       "      <td>1.063616</td>\n",
       "      <td>0.000139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1.057647</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>1.058086</td>\n",
       "      <td>0.000140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.052184</td>\n",
       "      <td>0.000055</td>\n",
       "      <td>1.052679</td>\n",
       "      <td>0.000189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1.046825</td>\n",
       "      <td>0.000089</td>\n",
       "      <td>1.047396</td>\n",
       "      <td>0.000211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1.041470</td>\n",
       "      <td>0.000074</td>\n",
       "      <td>1.042101</td>\n",
       "      <td>0.000205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.036222</td>\n",
       "      <td>0.000127</td>\n",
       "      <td>1.036908</td>\n",
       "      <td>0.000225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1.031058</td>\n",
       "      <td>0.000125</td>\n",
       "      <td>1.031802</td>\n",
       "      <td>0.000199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1.026017</td>\n",
       "      <td>0.000103</td>\n",
       "      <td>1.026828</td>\n",
       "      <td>0.000266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1.021022</td>\n",
       "      <td>0.000080</td>\n",
       "      <td>1.021890</td>\n",
       "      <td>0.000326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1.016149</td>\n",
       "      <td>0.000115</td>\n",
       "      <td>1.017086</td>\n",
       "      <td>0.000397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1.011310</td>\n",
       "      <td>0.000157</td>\n",
       "      <td>1.012300</td>\n",
       "      <td>0.000410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1.006557</td>\n",
       "      <td>0.000168</td>\n",
       "      <td>1.007594</td>\n",
       "      <td>0.000430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1.001816</td>\n",
       "      <td>0.000169</td>\n",
       "      <td>1.002913</td>\n",
       "      <td>0.000440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.997166</td>\n",
       "      <td>0.000178</td>\n",
       "      <td>0.998323</td>\n",
       "      <td>0.000464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.992588</td>\n",
       "      <td>0.000220</td>\n",
       "      <td>0.993802</td>\n",
       "      <td>0.000458</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.988180</td>\n",
       "      <td>0.000227</td>\n",
       "      <td>0.989449</td>\n",
       "      <td>0.000549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0.983755</td>\n",
       "      <td>0.000223</td>\n",
       "      <td>0.985084</td>\n",
       "      <td>0.000560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0.979376</td>\n",
       "      <td>0.000207</td>\n",
       "      <td>0.980761</td>\n",
       "      <td>0.000552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0.975056</td>\n",
       "      <td>0.000231</td>\n",
       "      <td>0.976506</td>\n",
       "      <td>0.000573</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0.970763</td>\n",
       "      <td>0.000232</td>\n",
       "      <td>0.972272</td>\n",
       "      <td>0.000590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0.966556</td>\n",
       "      <td>0.000247</td>\n",
       "      <td>0.968126</td>\n",
       "      <td>0.000606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.962432</td>\n",
       "      <td>0.000271</td>\n",
       "      <td>0.964078</td>\n",
       "      <td>0.000582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0.958367</td>\n",
       "      <td>0.000300</td>\n",
       "      <td>0.960076</td>\n",
       "      <td>0.000580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0.954382</td>\n",
       "      <td>0.000284</td>\n",
       "      <td>0.956157</td>\n",
       "      <td>0.000571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0.950453</td>\n",
       "      <td>0.000273</td>\n",
       "      <td>0.952288</td>\n",
       "      <td>0.000561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1970</th>\n",
       "      <td>0.510655</td>\n",
       "      <td>0.001232</td>\n",
       "      <td>0.587083</td>\n",
       "      <td>0.002666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1971</th>\n",
       "      <td>0.510619</td>\n",
       "      <td>0.001244</td>\n",
       "      <td>0.587078</td>\n",
       "      <td>0.002666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1972</th>\n",
       "      <td>0.510586</td>\n",
       "      <td>0.001237</td>\n",
       "      <td>0.587078</td>\n",
       "      <td>0.002668</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1973</th>\n",
       "      <td>0.510552</td>\n",
       "      <td>0.001236</td>\n",
       "      <td>0.587075</td>\n",
       "      <td>0.002667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1974</th>\n",
       "      <td>0.510518</td>\n",
       "      <td>0.001219</td>\n",
       "      <td>0.587074</td>\n",
       "      <td>0.002666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1975</th>\n",
       "      <td>0.510476</td>\n",
       "      <td>0.001217</td>\n",
       "      <td>0.587066</td>\n",
       "      <td>0.002665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1976</th>\n",
       "      <td>0.510444</td>\n",
       "      <td>0.001216</td>\n",
       "      <td>0.587060</td>\n",
       "      <td>0.002663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1977</th>\n",
       "      <td>0.510403</td>\n",
       "      <td>0.001212</td>\n",
       "      <td>0.587050</td>\n",
       "      <td>0.002665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1978</th>\n",
       "      <td>0.510368</td>\n",
       "      <td>0.001206</td>\n",
       "      <td>0.587046</td>\n",
       "      <td>0.002665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1979</th>\n",
       "      <td>0.510326</td>\n",
       "      <td>0.001212</td>\n",
       "      <td>0.587041</td>\n",
       "      <td>0.002664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1980</th>\n",
       "      <td>0.510281</td>\n",
       "      <td>0.001222</td>\n",
       "      <td>0.587025</td>\n",
       "      <td>0.002659</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1981</th>\n",
       "      <td>0.510244</td>\n",
       "      <td>0.001223</td>\n",
       "      <td>0.587018</td>\n",
       "      <td>0.002661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1982</th>\n",
       "      <td>0.510218</td>\n",
       "      <td>0.001229</td>\n",
       "      <td>0.587016</td>\n",
       "      <td>0.002666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1983</th>\n",
       "      <td>0.510194</td>\n",
       "      <td>0.001228</td>\n",
       "      <td>0.587007</td>\n",
       "      <td>0.002671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1984</th>\n",
       "      <td>0.510166</td>\n",
       "      <td>0.001236</td>\n",
       "      <td>0.587001</td>\n",
       "      <td>0.002669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1985</th>\n",
       "      <td>0.510117</td>\n",
       "      <td>0.001230</td>\n",
       "      <td>0.586992</td>\n",
       "      <td>0.002671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1986</th>\n",
       "      <td>0.510085</td>\n",
       "      <td>0.001234</td>\n",
       "      <td>0.586989</td>\n",
       "      <td>0.002670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1987</th>\n",
       "      <td>0.510053</td>\n",
       "      <td>0.001231</td>\n",
       "      <td>0.586987</td>\n",
       "      <td>0.002671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1988</th>\n",
       "      <td>0.510019</td>\n",
       "      <td>0.001230</td>\n",
       "      <td>0.586982</td>\n",
       "      <td>0.002676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1989</th>\n",
       "      <td>0.509989</td>\n",
       "      <td>0.001223</td>\n",
       "      <td>0.586980</td>\n",
       "      <td>0.002676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1990</th>\n",
       "      <td>0.509962</td>\n",
       "      <td>0.001215</td>\n",
       "      <td>0.586977</td>\n",
       "      <td>0.002674</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1991</th>\n",
       "      <td>0.509933</td>\n",
       "      <td>0.001208</td>\n",
       "      <td>0.586974</td>\n",
       "      <td>0.002675</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1992</th>\n",
       "      <td>0.509897</td>\n",
       "      <td>0.001197</td>\n",
       "      <td>0.586971</td>\n",
       "      <td>0.002679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1993</th>\n",
       "      <td>0.509856</td>\n",
       "      <td>0.001182</td>\n",
       "      <td>0.586962</td>\n",
       "      <td>0.002682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1994</th>\n",
       "      <td>0.509818</td>\n",
       "      <td>0.001176</td>\n",
       "      <td>0.586953</td>\n",
       "      <td>0.002685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1995</th>\n",
       "      <td>0.509784</td>\n",
       "      <td>0.001165</td>\n",
       "      <td>0.586947</td>\n",
       "      <td>0.002685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1996</th>\n",
       "      <td>0.509756</td>\n",
       "      <td>0.001164</td>\n",
       "      <td>0.586945</td>\n",
       "      <td>0.002686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1997</th>\n",
       "      <td>0.509716</td>\n",
       "      <td>0.001162</td>\n",
       "      <td>0.586937</td>\n",
       "      <td>0.002695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1998</th>\n",
       "      <td>0.509689</td>\n",
       "      <td>0.001161</td>\n",
       "      <td>0.586933</td>\n",
       "      <td>0.002697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1999</th>\n",
       "      <td>0.509664</td>\n",
       "      <td>0.001157</td>\n",
       "      <td>0.586930</td>\n",
       "      <td>0.002697</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2000 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      train-mlogloss-mean  train-mlogloss-std  test-mlogloss-mean  \\\n",
       "0                1.092398            0.000013            1.092467   \n",
       "1                1.086355            0.000006            1.086486   \n",
       "2                1.080416            0.000032            1.080607   \n",
       "3                1.074607            0.000057            1.074855   \n",
       "4                1.068885            0.000094            1.069194   \n",
       "5                1.063243            0.000013            1.063616   \n",
       "6                1.057647            0.000008            1.058086   \n",
       "7                1.052184            0.000055            1.052679   \n",
       "8                1.046825            0.000089            1.047396   \n",
       "9                1.041470            0.000074            1.042101   \n",
       "10               1.036222            0.000127            1.036908   \n",
       "11               1.031058            0.000125            1.031802   \n",
       "12               1.026017            0.000103            1.026828   \n",
       "13               1.021022            0.000080            1.021890   \n",
       "14               1.016149            0.000115            1.017086   \n",
       "15               1.011310            0.000157            1.012300   \n",
       "16               1.006557            0.000168            1.007594   \n",
       "17               1.001816            0.000169            1.002913   \n",
       "18               0.997166            0.000178            0.998323   \n",
       "19               0.992588            0.000220            0.993802   \n",
       "20               0.988180            0.000227            0.989449   \n",
       "21               0.983755            0.000223            0.985084   \n",
       "22               0.979376            0.000207            0.980761   \n",
       "23               0.975056            0.000231            0.976506   \n",
       "24               0.970763            0.000232            0.972272   \n",
       "25               0.966556            0.000247            0.968126   \n",
       "26               0.962432            0.000271            0.964078   \n",
       "27               0.958367            0.000300            0.960076   \n",
       "28               0.954382            0.000284            0.956157   \n",
       "29               0.950453            0.000273            0.952288   \n",
       "...                   ...                 ...                 ...   \n",
       "1970             0.510655            0.001232            0.587083   \n",
       "1971             0.510619            0.001244            0.587078   \n",
       "1972             0.510586            0.001237            0.587078   \n",
       "1973             0.510552            0.001236            0.587075   \n",
       "1974             0.510518            0.001219            0.587074   \n",
       "1975             0.510476            0.001217            0.587066   \n",
       "1976             0.510444            0.001216            0.587060   \n",
       "1977             0.510403            0.001212            0.587050   \n",
       "1978             0.510368            0.001206            0.587046   \n",
       "1979             0.510326            0.001212            0.587041   \n",
       "1980             0.510281            0.001222            0.587025   \n",
       "1981             0.510244            0.001223            0.587018   \n",
       "1982             0.510218            0.001229            0.587016   \n",
       "1983             0.510194            0.001228            0.587007   \n",
       "1984             0.510166            0.001236            0.587001   \n",
       "1985             0.510117            0.001230            0.586992   \n",
       "1986             0.510085            0.001234            0.586989   \n",
       "1987             0.510053            0.001231            0.586987   \n",
       "1988             0.510019            0.001230            0.586982   \n",
       "1989             0.509989            0.001223            0.586980   \n",
       "1990             0.509962            0.001215            0.586977   \n",
       "1991             0.509933            0.001208            0.586974   \n",
       "1992             0.509897            0.001197            0.586971   \n",
       "1993             0.509856            0.001182            0.586962   \n",
       "1994             0.509818            0.001176            0.586953   \n",
       "1995             0.509784            0.001165            0.586947   \n",
       "1996             0.509756            0.001164            0.586945   \n",
       "1997             0.509716            0.001162            0.586937   \n",
       "1998             0.509689            0.001161            0.586933   \n",
       "1999             0.509664            0.001157            0.586930   \n",
       "\n",
       "      test-mlogloss-std  \n",
       "0              0.000018  \n",
       "1              0.000043  \n",
       "2              0.000091  \n",
       "3              0.000055  \n",
       "4              0.000052  \n",
       "5              0.000139  \n",
       "6              0.000140  \n",
       "7              0.000189  \n",
       "8              0.000211  \n",
       "9              0.000205  \n",
       "10             0.000225  \n",
       "11             0.000199  \n",
       "12             0.000266  \n",
       "13             0.000326  \n",
       "14             0.000397  \n",
       "15             0.000410  \n",
       "16             0.000430  \n",
       "17             0.000440  \n",
       "18             0.000464  \n",
       "19             0.000458  \n",
       "20             0.000549  \n",
       "21             0.000560  \n",
       "22             0.000552  \n",
       "23             0.000573  \n",
       "24             0.000590  \n",
       "25             0.000606  \n",
       "26             0.000582  \n",
       "27             0.000580  \n",
       "28             0.000571  \n",
       "29             0.000561  \n",
       "...                 ...  \n",
       "1970           0.002666  \n",
       "1971           0.002666  \n",
       "1972           0.002668  \n",
       "1973           0.002667  \n",
       "1974           0.002666  \n",
       "1975           0.002665  \n",
       "1976           0.002663  \n",
       "1977           0.002665  \n",
       "1978           0.002665  \n",
       "1979           0.002664  \n",
       "1980           0.002659  \n",
       "1981           0.002661  \n",
       "1982           0.002666  \n",
       "1983           0.002671  \n",
       "1984           0.002669  \n",
       "1985           0.002671  \n",
       "1986           0.002670  \n",
       "1987           0.002671  \n",
       "1988           0.002676  \n",
       "1989           0.002676  \n",
       "1990           0.002674  \n",
       "1991           0.002675  \n",
       "1992           0.002679  \n",
       "1993           0.002682  \n",
       "1994           0.002685  \n",
       "1995           0.002685  \n",
       "1996           0.002686  \n",
       "1997           0.002695  \n",
       "1998           0.002697  \n",
       "1999           0.002697  \n",
       "\n",
       "[2000 rows x 4 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#对 xgboost 进行配置\n",
    "xgb2 = XGBClassifier(\n",
    "        learning_rate =0.01,\n",
    "        n_estimators=2000,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=xgb1.max_depth,\n",
    "        min_child_weight=xgb1.min_child_weight,\n",
    "        gamma=0,\n",
    "        subsample=xgb1.subsample,\n",
    "        colsample_bytree=xgb1.colsample_bytree,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "xgb_param2 = xgb2.get_xgb_params()\n",
    "#三中分类的问题，把num_class 设置为3\n",
    "xgb_param2['num_class'] = 3\n",
    "#通过xgboost 中内置的交叉验证选择初始的若学习期数目\n",
    "cvresult_round2 = xgb.cv(xgb_param2, xgtrain, num_boost_round=xgb2.get_params()['n_estimators'], folds =kfold,metrics='mlogloss', early_stopping_rounds=50)\n",
    "# 将结果保存起来\n",
    "cvresult_round2.to_csv('xgBoostRentalCV_Result2.csv', index_label = 'n_estimators')\n",
    "cvresult_round2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5276916595089363\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jack/anaconda3/lib/python3.6/site-packages/sklearn/preprocessing/label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    }
   ],
   "source": [
    "#设置最优弱学习器\n",
    "xgb2.n_estimators = cvresult_round2.shape[0]\n",
    "# 根据xgb的cv后最优参数对模型进行训练\n",
    "xgb2.fit(x_train, y_train, eval_metric='mlogloss')\n",
    "        \n",
    "#P输出评价得分:\n",
    "predict_proba2 = xgb2.predict_proba(x_train)\n",
    "predict_result = xgb2.predict(x_train)\n",
    "logloss2 = log_loss(y_train, predict_proba2)\n",
    "print(logloss2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_estimators: 376\n",
      "Accuracy: 0.7758145566542389\n"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "print('n_estimators:', n_estimators)\n",
    "print('Accuracy:',metrics.accuracy_score(y_train, predict_result))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 调用模型进行测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jack/anaconda3/lib/python3.6/site-packages/sklearn/preprocessing/label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    }
   ],
   "source": [
    "predict_test_result = xgb2.predict(data_test)\n",
    "predict_test_probability =  xgb2.predict_proba(data_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test =pd.DataFrame(predict_test_result,columns=[\"interest_level\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test[\"interest_level\"] = y_test[\"interest_level\"].map({0: 'low', 1: 'medium', 2: 'high'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74629</th>\n",
       "      <td>medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74630</th>\n",
       "      <td>medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74631</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74632</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74633</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74634</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74635</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74636</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74637</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74638</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74639</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74640</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74641</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74642</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74643</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74644</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74645</th>\n",
       "      <td>medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74646</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74647</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74648</th>\n",
       "      <td>medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74649</th>\n",
       "      <td>medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74650</th>\n",
       "      <td>medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74651</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74652</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74653</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74654</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74655</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74656</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74657</th>\n",
       "      <td>low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74658</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>74659 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      interest_level\n",
       "0               high\n",
       "1             medium\n",
       "2               high\n",
       "3               high\n",
       "4               high\n",
       "5               high\n",
       "6               high\n",
       "7             medium\n",
       "8               high\n",
       "9               high\n",
       "10              high\n",
       "11              high\n",
       "12              high\n",
       "13              high\n",
       "14              high\n",
       "15              high\n",
       "16              high\n",
       "17              high\n",
       "18              high\n",
       "19              high\n",
       "20              high\n",
       "21              high\n",
       "22              high\n",
       "23            medium\n",
       "24              high\n",
       "25            medium\n",
       "26              high\n",
       "27            medium\n",
       "28              high\n",
       "29              high\n",
       "...              ...\n",
       "74629         medium\n",
       "74630         medium\n",
       "74631           high\n",
       "74632           high\n",
       "74633           high\n",
       "74634           high\n",
       "74635           high\n",
       "74636           high\n",
       "74637           high\n",
       "74638           high\n",
       "74639           high\n",
       "74640           high\n",
       "74641           high\n",
       "74642           high\n",
       "74643           high\n",
       "74644           high\n",
       "74645         medium\n",
       "74646           high\n",
       "74647           high\n",
       "74648         medium\n",
       "74649         medium\n",
       "74650         medium\n",
       "74651           high\n",
       "74652           high\n",
       "74653           high\n",
       "74654           high\n",
       "74655           high\n",
       "74656           high\n",
       "74657            low\n",
       "74658           high\n",
       "\n",
       "[74659 rows x 1 columns]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 生成测试结果文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test.to_csv(\"rent_ListForPredict.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_probability = pd.DataFrame(predict_test_probability,columns=[\"low\", \"medium\", \"high\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>low</th>\n",
       "      <th>medium</th>\n",
       "      <th>high</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.090066</td>\n",
       "      <td>0.394855</td>\n",
       "      <td>0.515079</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.242824</td>\n",
       "      <td>0.405843</td>\n",
       "      <td>0.351332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.041978</td>\n",
       "      <td>0.108655</td>\n",
       "      <td>0.849367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.058809</td>\n",
       "      <td>0.331764</td>\n",
       "      <td>0.609427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.081306</td>\n",
       "      <td>0.250564</td>\n",
       "      <td>0.668130</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        low    medium      high\n",
       "0  0.090066  0.394855  0.515079\n",
       "1  0.242824  0.405843  0.351332\n",
       "2  0.041978  0.108655  0.849367\n",
       "3  0.058809  0.331764  0.609427\n",
       "4  0.081306  0.250564  0.668130"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_probability.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_probability.to_csv(\"rent_ListForPredict_prob.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
